# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from py09_1.demo.demo.items import Qiancheng
import datetime
from w3lib.html import remove_tags


class QianchengSpider(CrawlSpider):
    name = 'qiancheng'
    allowed_domains = ['51job.com']
    start_urls = ['http://51job.com/']

    custom_settings = {
        'DEFAULT_REQUEST_HEADERS':{
            'Host': ' search.51job.com',
            'Connection': ' keep-alive',
            'Cache-Control': ' max-age=0',
            'Upgrade-Insecure-Requests': ' 1',
            'User-Agent': ' Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36',
            'Accept': ' text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
            'Referer': ' https://search.51job.com/list/010000,000000,0000,00,9,99,%2B,2,3.html?lang=c&stype=1&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=',
            'Accept-Encoding': ' gzip, deflate, br',
            'Accept-Language': ' zh-CN,zh;q=0.9',
            'Cookie': ' adv=adsnew%3D1%26%7C%26adsresume%3D1%26%7C%26adsfrom%3Dhttps%253A%252F%252Fwww.baidu.com%252Fbaidu.php%253Fsc.060000jZYuibWkd0MBdlzSzXmnBPyD76r9WFGUbfS-KdNt9qgXqB2up6cxXNAIqiNV1c0odnT-eP1Rbg8PXrZQDLC3ClWXfcUg7fi34RYEiNKKuu3KERTKgsAiSfXooGgeVcok1tfgxesFyUJASGpkEFe4Ks3J8-xwQF9VYubjMJa2ofVf.7R_NR2Ar5Od66CHnsGtVdXNdlc2D1n2xx81IZ76Y_NvNeJtheJ1-k_nOEOlecxLO3MHSEwECntx135zOCxgvg45E6OeAHxfOgkOdkxo3O-CLOWEWEzgxwOsr5Oml5dlpoQOvSajSw7OVxwxS9yOO_xVYXveqElqEgVmvfdG_H3en-dvHFIjxvuQVOB-MFb8lRq5uEtNPHV2XgZJyAp7WWELI-p6.U1Yk0ZDqkea11neXYtT0TA-W5HD0IjLZ_Q5n1VeHksKGUHYznjf0u1dBugK1nfKdpHdBmy-bIfKspyfqP0KWpyfqrHn0UgfqnH0kndtknjD4g1DsnHIxnH0YP-t1PW0k0AVG5H00TMfqrjbs0ANGujYzPWRzndtkPjnkg1cknHD1g1D3PHmsg1csrHns0AFG5HcsP0KVm1YknHb4rHcvPHwxnH0snNtLPH6dnjT3rH-xnH0zg100TgKGujYs0Z7Wpyfqn0KzuLw9u1Ys0A7B5HKxn0K-ThTqn0KsTjYknHfdn104n1Ts0A4vTjYsQW0snj0snj0s0AdYTjYs0AwbUL0qn0KzpWYs0Aw-IWdsmsKhIjYs0ZKC5H00ULnqn0KBI1Ykn0K8IjYs0ZPl5fKYIgnqnHc1rHf1PjDsnWb1njbvnHRLnjb0ThNkIjYkPHnsPHD1P16Lnjc40ZPGujdhryfvmHcvuH0snjcvuhDk0AP1UHY4nDPDrDRkwH0drDfdPjT10A7W5HD0TA3qn0KkUgfqn0KkUgnqn0KlIjYs0AdWgvuzUvYqn7tsg1Kxn7ts0Aw9UMNBuNqsUA78pyw15HKxn7tsg1nkP16YnNts0ZK9I7qhUA7M5H00uAPGujYs0ANYpyfqQHD0mgPsmvnqn0KdTA-8mvnqn0KkUymqn0KhmLNY5H00uMGC5H00uh7Y5H00XMK_Ignqn0K9uAu_myTqnfK_uhnqn0KWThnqPj6snjD%2526ck%253D6614.4.87.388.427.294.507.145%2526shh%253Dwww.baidu.com%2526sht%253Dbaiduhome_pg%2526us%253D1.0.1.0.2.777.0%2526ie%253Dutf-8%2526f%253D8%2526tn%253Dbaiduhome_pg%2526wd%253D%2525E5%252589%25258D%2525E7%2525A8%25258B%2525E6%252597%2525A0%2525E5%2525BF%2525A7%2526oq%253D%252525E6%25252599%252525BA%252525E8%25252581%25252594%252525E6%2525258B%2525259B%252525E8%25252581%25252598%252525E7%25252588%252525AC%252525E8%25252599%252525AB%2526rqlang%253Dcn%2526inputT%253D118971%2526bc%253D110101%26%7C%26adsnum%3D1015125; guid=15305137953848290061; partner=www_baidu_com; 51job=cenglish%3D0%26%7C%26; nsearch=jobarea%3D%26%7C%26ord_field%3D%26%7C%26recentSearch0%3D%26%7C%26recentSearch1%3D%26%7C%26recentSearch2%3D%26%7C%26recentSearch3%3D%26%7C%26recentSearch4%3D%26%7C%26collapse_expansion%3D; slife=lastvisit%3D010000%26%7C%26; search=jobarea%7E%60010000%7C%21ord_field%7E%600%7C%21recentSearch0%7E%601%A1%FB%A1%FA010000%2C00%A1%FB%A1%FA000000%A1%FB%A1%FA0000%A1%FB%A1%FA00%A1%FB%A1%FA9%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA2%A1%FB%A1%FA%A1%FB%A1%FA-1%A1%FB%A1%FA1530515409%A1%FB%A1%FA0%A1%FB%A1%FA%A1%FB%A1%FA%7C%21recentSearch1%7E%601%A1%FB%A1%FA070000%2C00%A1%FB%A1%FA000000%A1%FB%A1%FA0000%A1%FB%A1%FA26%A1%FB%A1%FA1%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA%CF%FA%CA%DB%2F%CA%D0%B3%A1%A1%FB%A1%FA2%A1%FB%A1%FA%A1%FB%A1%FA-1%A1%FB%A1%FA1530516701%A1%FB%A1%FA0%A1%FB%A1%FA%A1%FB%A1%FA%7C%21recentSearch2%7E%601%A1%FB%A1%FA070000%2C00%A1%FB%A1%FA000000%A1%FB%A1%FA0000%A1%FB%A1%FA26%A1%FB%A1%FA9%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA%B2%C6%CE%F1%BE%AD%C0%ED%A1%FB%A1%FA2%A1%FB%A1%FA%A1%FB%A1%FA-1%A1%FB%A1%FA1530516373%A1%FB%A1%FA0%A1%FB%A1%FA%A1%FB%A1%FA%7C%21recentSearch3%7E%601%A1%FB%A1%FA070000%2C00%A1%FB%A1%FA000000%A1%FB%A1%FA0000%A1%FB%A1%FA26%A1%FB%A1%FA9%A1%FB%A1%FA99%A1%FB%A1%FA11%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA%CF%FA%CA%DB%2F%CA%D0%B3%A1%A1%FB%A1%FA2%A1%FB%A1%FA%A1%FB%A1%FA-1%A1%FB%A1%FA1530516448%A1%FB%A1%FA0%A1%FB%A1%FA%A1%FB%A1%FA%BC%BE%B6%C8%BD%B1%BD%F0%7C%21recentSearch4%7E%601%A1%FB%A1%FA070000%2C00%A1%FB%A1%FA000000%A1%FB%A1%FA0000%A1%FB%A1%FA26%A1%FB%A1%FA0%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA%CF%FA%CA%DB%2F%CA%D0%B3%A1%A1%FB%A1%FA2%A1%FB%A1%FA%A1%FB%A1%FA-1%A1%FB%A1%FA1530516409%A1%FB%A1%FA0%A1%FB%A1%FA%A1%FB%A1%FA%7C%21collapse_expansion%7E%601%7C%21',
        },
        'COOKIES_ENABLED': False,
    }

    rules = (
        # Rule(LinkExtractor(allow=r'search.51job.com/list/010000,000000,0000,00,2?,.+.html'),follow=True),
        Rule(LinkExtractor(allow=r'jobs.51job.com/beijing-.*/\d+.html'),callback='parse_item',follow=True),

    )

    def parse_item(self, response):
        item = Qiancheng()
        url = response.url  #  链接
        title = response.xpath('//div[@class="cn"]/h1/text()').extract()[0]  # 标题
        data = []
        slarys = response.css('div.cn strong::text').extract()
        if slarys == data:
            slarys  = '面议'
        else:
            slarys = slarys[0]
        slary = slarys   # 薪资
        area = response.css('div.cn span::text').extract()[0]    #区域
        exp = response.css('div.tCompany_main span::text').extract()[0]    # 经验
        date = response.css('div.tCompany_main span::text').extract()[3]
        if '发布' not in date:
            date = '最近'
        else:
            date =  '2018-'+ date
        date_pub = date.replace('发布','')    # 发布时间
        advantages = response.css('.t2 span::text').extract()
        advantage = ','.join(advantages)   # 职位诱惑
        ptype = response.css('div.mt10 span::text').extract()[1]   # 职业类别
        jobdescs = response.xpath('/html/body/div[3]/div[2]/div[3]/*[2]').extract()
        for job in jobdescs:
            job = remove_tags(job).replace('\n','').replace('\t','').replace('\r','').replace('分享微信邮件','')
        jobdesc = job   # 岗位描述
        company = response.xpath('//p[@class="cname"]/a/text()').extract()[0]   # 公司

        edus = response.xpath('//div[@class="t1"]/span/text()').extract()[1]    # 学历
        if '招' in edus:
            edus = '学历不限'
        else:
            edus = edus

        edu = edus#  学历


        i = []
        jobaddrs = response.xpath('/html/body/div[3]/div[2]/div[3]/div[3]/div/p/text()').extract()[1].strip() # 工作地点
        if jobaddrs == i:
            jobaddrs = ''
        else:
            jobaddrs = jobaddrs
        jobaddr = jobaddrs
        crawl_time = datetime.datetime.now().strftime('%Y-%m-%d')



        item['url'] = url
        item['title'] = title
        item['slary'] = slary
        item['area'] = area
        item['exp'] = exp
        item['date_pub'] = date_pub
        item['advantage'] = advantage
        item['ptype'] = ptype
        item['jobdesc'] = jobdesc
        item['company'] = company
        item['edu'] = edu
        item['jobaddr'] = jobaddr
        item['crawl_time'] = crawl_time
        yield item




















